# -*- coding: UTF-8 -*-
# author-guanfeng_yang
#使用shutil进行多线程处理文件迁移。将某一位置的文件备份迁移到另一存储位置，可重复执行，存在的文件则跳过(只对文件名进行了比较，文件的正确性及完整性未比较)

import logging
import os
import shutil
from concurrent.futures import ThreadPoolExecutor
from logging.handlers import RotatingFileHandler
#原始存储位置
source_path = "D:\\fengxian"
#目标存储位置
target_path = "D:\\test_data"

logger = logging.getLogger("data_move")
logger.setLevel(logging.INFO)
fh = logging.handlers.RotatingFileHandler("data_move.log", maxBytes=10 * 1024 * 1024, backupCount=10)
fh.setLevel(logging.INFO)
out = logging.StreamHandler()
out.setLevel(logging.INFO)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
out.setFormatter(formatter)
fh.setFormatter(formatter)
logger.addHandler(out)
logger.addHandler(fh)


# ----------------list and copy-------------------#
# 遍历所有文件(不含文件夹)
def list_all_file(root, path, save):
    if os.path.isdir(path):
        for d in os.listdir(path):
            list_all_file(root, os.path.join(path, d), save)
    else:
        save.append(path.replace(root, ""))


# 找出source里在target不存在的文件列表
def find_not_exist(source, target):
    source_list = []
    target_list = []
    list_all_file(source, source, source_list)
    list_all_file(target, target, target_list)
    return list(set(source_list).difference(set(target_list)))


# copy source里在target不存在的文件到target
def copy_not_exist_file(source_file, target_file):
    target_file_parent = os.path.dirname(target_file)
    if not os.path.exists(target_file_parent):
        os.makedirs(target_file_parent)
    if not os.path.exists(target_file):
        shutil.copyfile(source_file, target_file)
        file_size = os.path.getsize(source_file)
        log_file_size = file_size
        if file_size < 1024:
            log_file_size = str(file_size) + "B"
        elif file_size > 1024:
            log_file_size = str(file_size / 1024) + "KB"
        else:
            log_file_size = str(file_size / 1024 / 1024) + "MB"
        logger.info("#### copy file path :%s | size:%s", target_file, log_file_size)


# execute copy max_workers：线程池大小
def execute_copy(source, target, max_workers):
    list_file = find_not_exist(source, target)
    # 开启多线程进行copy文件
    executor = ThreadPoolExecutor(max_workers=max_workers)
    for p in list_file:
        source_file = source + p
        target_file = target + p
        executor.submit(copy_not_exist_file, source_file, target_file)


# ------------fast copy ---------------#
# 遍历及对比copy max_workers：线程池大小
def fast_copy(source, target, max_workers):
    executor = ThreadPoolExecutor(max_workers=max_workers)
    for root, dirs, files in os.walk(source):
        for f in files:
            source_file = os.path.join(root, f)
            target_file = source_file.replace(source, target)
            executor.submit(copy, source_file, target_file)


#
def copy(source_file, target_file):
    file_size = os.path.getsize(source_file)
    if os.path.exists(target_file):
        target_file_size = os.path.getsize(target_file)
        if file_size == target_file_size:
            return
        os.remove(target_file)
    # file not exist
    target_file_parent = os.path.dirname(target_file)
    if not os.path.exists(target_file_parent):
        os.makedirs(target_file_parent)

    shutil.copyfile(source_file, target_file)
    log_file_size = file_size
    if file_size < 1024:
        log_file_size = str(file_size) + "B"
    elif (file_size > 1024) and (file_size < 1024 * 1024):
        log_file_size = str(file_size / 1024) + "KB"
    else:
        log_file_size = str(file_size / 1024 / 1024) + "MB"
    logger.info("#### copy file path :%s | size:%s", target_file, log_file_size)


if __name__ == "__main__":
    # execute_copy(source_path, target_path, 10)
    fast_copy(source_path, target_path, 10)
